﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using ParseReuters21578;

namespace MyTestProject
{
    enum FeatureSelectionMethod
    {
        AllFeatures, IDF500TO1000, ClosestToMean
    }
    
    class Program
    {
        

        static void Main(string[] args)
        {
            List<ICheckable> dataset = Reuters21578.ReadFiles(true, true, true);

            //List<ICheckable> datasetModApte = Reuters21578.GetCompleteModApte(dataset);

            //dataset = datasetModApte;

            Dictionary<string, List<ICheckable>> ds = Reuters21578.GetWantedTopicsSingleItems(dataset);
            List<ICheckable> nds = new List<ICheckable>();
            foreach (var topic in ds)
            {
                foreach (var item in topic.Value)
                {
                    nds.Add(item);
                }
            }

            dataset = nds;

            Dictionary<string, double> idf = IDF(dataset);

            List<string> features = FeatureSelection(dataset, idf, FeatureSelectionMethod.IDF500TO1000);

            ArffWriter.WriteTFIDF("tf-wanted-topics.arff", dataset, idf, features);

            #region Export a term-frequency-in-documents.csv
            //Dictionary<int, int> termNumberDist = new Dictionary<int, int>();
            //foreach (ICheckable item in dataset)
            //{
            //    if (item.TF.Keys.Count == 1)
            //    {
            //    }
            //    if (termNumberDist.ContainsKey(item.TF.Keys.Count))
            //    {
            //        termNumberDist[item.TF.Keys.Count]++;
            //    }
            //    else
            //    {
            //        termNumberDist.Add(item.TF.Keys.Count, 1);
            //    }
            //}

            //System.IO.StreamWriter writer = System.IO.File.CreateText("term-frequency-in-documents-mod-apte.csv");
            //foreach (var item in termNumberDist)
            //{
            //    writer.WriteLine(item.Key + "," + item.Value);
            //}
            //writer.Close();
            #endregion

            #region Export the idf table
            //System.IO.StreamWriter writer2 = System.IO.File.CreateText("idf-mod-apte.csv");
            //writer2.WriteLine("term,idf");
            //foreach (var term in idf)
            //{
            //    writer2.WriteLine(term.Key + "," + term.Value);
            //}
            //writer2.Close();
            #endregion

            //System.IO.StreamWriter writer = System.IO.File.CreateText("tf-idf-reuters-all.txt"); 
            //ExportTFIDF_ForBigDatasets(writer, dataset, idf, features);
            //writer.Close();

            //double[,] simMatrix = GetSimilarityMarix(dataset);
            //Console.WriteLine(PrintCosSimilarity(simMatrix));
        }

        private static List<string> FeatureSelection(List<ICheckable> dataset, Dictionary<string, double> idf, FeatureSelectionMethod featureSelectionMethod)
        {
            List<string> features = new List<string>();
            switch (featureSelectionMethod)
            {
                case FeatureSelectionMethod.AllFeatures:
                    features = idf.Keys.ToList();
                    break;
                case FeatureSelectionMethod.IDF500TO1000:
                    List<KeyValuePair<string, double>> myList = idf.ToList();
                    myList.Sort((x, y) => x.Value.CompareTo(y.Value));
                    for (int i = 500; i < 1000; i++)
                    {
                        features.Add(myList[i].Key);
                    }
                    break;
                case FeatureSelectionMethod.ClosestToMean:
                    break;
            }
            return features;
        }

        private static void ExportTFIDF_ForBigDatasets(StreamWriter writer, List<ICheckable> dataset, Dictionary<string, double> idf, List<string> features)
        {
            double tf;
            foreach (ICheckable item in dataset)
            {
                writer.Write(item.ID);
                foreach (string feature in features)
                {
                    if(item.TF.ContainsKey(feature))
                        tf = item.TF[feature];
                    else
                        tf = 0;

                    writer.Write(',' + (tf * idf[feature]));
                }
                writer.WriteLine();
            }
        }

        static double[,] GetSimilarityMarix(List<ICheckable> dataset)
        {
            int n = dataset.Count;
            double[,] simMatrix = new double[n, n];

            for (int i = 0; i < n; i++)
            {
                for (int j = 0; j < n; j++)
                {
                    simMatrix[i, j] = CosineSimilarity(dataset[i], dataset[j]);
                }
            }
            return simMatrix;
        }

        static string PrintCosSimilarity(double[,] simMatrix)
        {
            string toPrint = string.Empty;
            for (int i = 0; i < simMatrix.GetLength(0); i++)
            {
                toPrint += "\t" + i;
            }
            toPrint += Environment.NewLine;
            for (int i = 0; i < simMatrix.GetLength(0); i++)
            {
                toPrint += i;
                for (int j = 0; j < simMatrix.GetLength(0); j++)
                {
                    toPrint += "\t" + simMatrix[i, j].ToString("#.000");
                }
                toPrint += Environment.NewLine;
            }
            return toPrint;
        }

        static string PrintTF(List<Document> dataset)
        {
            string ans = string.Empty;
            foreach (Document document in dataset)
            {
                ans += document.ToString();
            }
            return ans;
        }

        static string PrintTFIDF_ForShortDatasets(List<ICheckable> dataset, Dictionary<string, double> idf)
        {
            string toPrint = string.Empty;
            toPrint += "Term";
            foreach (string word in idf.Keys)
            {
                toPrint += "\t" + word;
            }
            toPrint += Environment.NewLine + "Item" + Environment.NewLine;

            foreach (ICheckable item in dataset)
            {
                foreach (string word in idf.Keys)
                {
                    toPrint += "\t" + TFIDF(word, item, idf).ToString("#.000");
                }
                toPrint += Environment.NewLine;
            }

            return toPrint;
        }

        static double CosineSimilarity(ICheckable a, ICheckable b)
        {
            double ans = 0;
            List<string> miniLexicon = new List<string>(a.TF.Keys);
            foreach (string key in b.TF.Keys)
            {
                if (!miniLexicon.Contains(key))
                {
                    miniLexicon.Add(key);
                }
            }

            double[] vecA = new double[miniLexicon.Count];
            double[] vecB = new double[miniLexicon.Count];

            double ab = 0, aPow = 0, bPow = 0;
            for (int i = 0; i < miniLexicon.Count; i++)
            {
                vecA[i] = (double)a.NumberOfOccurrences(miniLexicon[i]);
                vecB[i] = (double)b.NumberOfOccurrences(miniLexicon[i]);
                ab += vecA[i] * vecB[i];
                aPow += Math.Pow(vecA[i], 2);
                bPow += Math.Pow(vecB[i], 2);
            }

            ans = ab / (Math.Sqrt(aPow) * Math.Sqrt(bPow));

            return ans;
        }

        static Dictionary<string, double> IDF(List<ICheckable> dataset)
        {
            bool exportLexicon = false;
            System.IO.StreamWriter writer = null;

            List<string> lexicon = new List<string>();
            Dictionary<string, double> idf = new Dictionary<string, double>();

            if(exportLexicon)
                writer = System.IO.File.CreateText("lexicon.txt");

            foreach (ICheckable item in dataset)
            {
                string[] docArray = item.Text.Split(' ');
                foreach (string word in docArray)
                {
                    if (!lexicon.Contains(word))
                    {
                        lexicon.Add(word);
                        if (exportLexicon && writer != null)
                            writer.WriteLine(word);
                    }
                }
            }
            if (exportLexicon)
                writer.Close();

            #region Option 1
            //double numOfOccurencesInDataset;
            //foreach (string word in lexicon)
            //{
            //    numOfOccurencesInDataset = 0;
            //    foreach (ICheckable item in dataset)
            //    {
            //        if (item.Contains(word))
            //        {
            //            numOfOccurencesInDataset++;
            //        }
            //    }
            //    idf.Add(word, Math.Log(Math.Abs((double)dataset.Count / numOfOccurencesInDataset), 2));
            //}
            #endregion

            #region Option 2
            Dictionary<string, double> numOfOccurencesInDataset = new Dictionary<string, double>();

            foreach (ICheckable item in dataset)
            {
                foreach (string word in item.TF.Keys)
                {
                    if (numOfOccurencesInDataset.ContainsKey(word))
                    {
                        numOfOccurencesInDataset[word]++;
                    }
                    else
                    {
                        numOfOccurencesInDataset.Add(word, 1);
                    }
                }
            }
            
            foreach (string word in lexicon)
            {
                idf.Add(word, Math.Log(Math.Abs((double)dataset.Count / numOfOccurencesInDataset[word]), 2));
                if (word == "dlr")
                {
                }
            }
            

            #endregion
            
            return idf;
        }

        static double TFIDF(string word, ICheckable item, Dictionary<string, double> idf)
        {
            double ans;
            if (!item.Contains(word))
            {
                ans = 0;
            }
            else
            {
                ans = item.TF[word] * idf[word];
            }
            return ans;
        }
    }

    static class IDMaker
    {
        static int _seed = 0;
        static public int GetUID()
        {
            return _seed++;
        }
    }
}
